{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pima-Indians——Logistic 回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.866045       -0.031990   \n",
       "1  -0.844885                     -1.205066       -0.528319   \n",
       "2   1.233880                      2.016662       -0.693761   \n",
       "3  -0.844885                     -1.073567       -0.528319   \n",
       "4  -1.141852                      0.504422       -2.679076   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.670643      -0.181541  0.166619   \n",
       "1                    -0.012301      -0.181541 -0.852200   \n",
       "2                    -0.012301      -0.181541 -1.332500   \n",
       "3                    -0.695245      -0.540642 -0.633881   \n",
       "4                     0.670643       0.316566  1.549303   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# 请自行在log(x+1)特征和tf_idf特征上尝试，并比较不同特征的结果，\n",
    "# path to where the data lies\n",
    "train = pd.read_csv(\"FE_pima-indians-diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']   \n",
    "X_train = train.drop([\"Target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "#查看一个学习器是否支持稀疏数据，可以看fit函数是否支持: X: {array-like, sparse matrix}.\n",
    "#可自行用timeit比较稠密数据和稀疏数据的训练时间\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5折交叉验证Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.48797856 0.53011593 0.4562292  0.422546   0.48392885]\n",
      "cv logloss is: 0.47615970944434044\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/surveyzhang/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/surveyzhang/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/surveyzhang/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/surveyzhang/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/home/surveyzhang/.local/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "#采用5折交叉验证\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "#%timeit loss_sparse = cross_val_score(lr, X_train_sparse, y_train, cv=3, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression + GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） \n",
    "目标函数为：J =  C* sum(logloss(f(xi), yi)) + penalty \n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同：\n",
    "1. 设置参数搜索范围\n",
    "2. 生成学习器实例（参数设置）\n",
    "3. 生成GridSearchCV的实例（参数设置）\n",
    "4. 调用GridSearchCV的fit方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='liblinear',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=4,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss',n_jobs = 4,)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.47602682979368166\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/surveyzhang/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/surveyzhang/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmUVPWd9/H3t6qavZGlmxZZ0mAwUUAwtkpccIkLMQmK+yTG6MQhzsiY+GRTk0MUZzIu4+TMPPqMC6NjlgkaHWOHkGA0IYiJCBIUEBckLg2yC8jSS3V9nz/u7aa6qe5b3dTtBT6vc+rUXX637reudH282++auyMiItKaRGcXICIiXZ/CQkREIiksREQkksJCREQiKSxERCSSwkJERCIpLEREJJLCQkREIiksREQkUqqzCyiUkpISLy8v7+wyRES6lZdffnmLu5dGtTtowqK8vJylS5d2dhkiIt2Kmb2bTzsdhhIRkUgKCxERiaSwEBGRSAfNOQsROXTV1dVRVVVFdXV1Z5fSZfXq1Yvhw4dTVFTUruUVFiLS7VVVVVFcXEx5eTlm1tnldDnuztatW6mqqmLUqFHt+gwdhhKRbq+6uprBgwcrKFpgZgwePPiA9rwUFiJyUFBQtO5At4/CQkQOSZc/8Gcuf+DPnV1Gt6GwEJEu5aRHLuakRy7u7DLarF+/fo3DU6ZMYcCAAXz+85/P2fb6669n4sSJHHPMMfTu3ZuJEycyceJEnnjiiTatc9myZTz42MOs3rLmgGrPh05wi8Ss4Ydv8TVPdnIl0lG+/e1vs2fPHh544IGc8++77z4A3nnnHT7/+c+zfPnydq1n2bJlLFqyiNM+M7ndteZLexYiIgX2mc98huLi4nYt+9Zbb3Heeedx/PHHM3nyZN58800A5syZw7hx45gwYQJnnnkme/fuZdasWcx9spJpZ3yhzXslbaU9C2mza357DQCPTHmkkysR2d9tv1rFa+t3RrZ77YOgTT7nLY45oj8/+MLYA64tH9OnT2f27NkceeSRvPDCC8yYMYNnnnmG2267jQULFlBWVsb27dvp3bs3M2fOZOGSRdz8z9/n6JKPx1pXrGFhZlOAfweSwGx3v6PZ/KuBu4F14aR73X12OO+3wCRgkbvnPvAnnaLhj0xECmv79u28+OKLXHzxvnM26XQagFNOOYWrrrqKSy+9lIsuuqjDa4stLMwsCdwHnANUAUvMrNLdX2vW9DF3n5HjI+4G+gBfi6tGETn45LsH0LBH8djXPh1nOW3i7pSUlOQ8h/HQQw+xePFi5s6dy4QJE3j11Vc7tLY4z1mcCKxx97XuXgvMAS7Id2F3fw74KK7iRES6moEDBzJ06FCeeuopADKZDK+88goAa9euZdKkSdx+++0MHDiQdevWUVxczO5duzuktjjDYhjwftZ4VTituYvN7FUze8LMRsRYj4hIhzjttNO49NJLee655xg+fDjz58/Pe9k5c+Zw//33M2HCBMaOHcvcuXMBuPHGGxk/fjzjx4/n7LPPZty4cZx11lm8sep1Ljpzarc+wZ3rdkFvNv4r4OfuXmNm1wGPAmflvQKz6cB0gJEjR7a3ThGRA7Zr167G4eeffz6vZcrLy1m5cmWTaaNHj84ZLpWVlftNKy0t5RfPBnsh3fkEdxWQvacwHFif3cDdt2aNPgTc2ZYVuPuDwIMAFRUVzYNIRKRFXelcRXcQ52GoJcAYMxtlZj2AK4Am0WhmQ7NGpwKrY6ynRbrtX0SkdbHtWbh72sxmAPMJLp192N1XmdksYKm7VwI3mNlUIA1sA65uWN7Mngc+CfQzsyrgq+6e/4E/EREpmFjvs3D3ecC8ZtNmZg3fDNzcwrKnxVmbiIjkT3dwA+/0+NdwSH33iIjkor6hROTQ9MjngpfkRWEhIlIAHd1F+VNPPcXdd999wHXnS4ehREQKrFBdlKfTaVKp3D/T06ZNA+iQZ1mA9ixERAruQLooP/XUU/ne977H5MmTuffee3n66ac56aSTOO644zj33HPZtGkTALNnz+Yb3/gGAN/5+2/y9a9/nZNPPpnRo0c3dhdSSNqzEJGDy29ugg0rotttCDviy+e8xeHj4bN3RLcrkJ07d7Jw4UIAPvzwQ6ZOnYqZcf/993PPPfdw553737+8adMmXnjhBVasWMFll13WuOdRKAoLEZEu5oorrmgcfu+997jsssvYsGEDNTU1HHXUUTmXufDCCzEzjj32WNatW5ezzYFQWIjIwSXfPYCGPYprfh1fLe3Ut2/fxuHrr7+eW265hfPPP59nn32WO+7I/f169uzZOOxe+N6PdM5CRKQL27FjB8OGDcPdefTRRzutDoWFiEiBHUgX5c3deuutTJs2jdNPP52ysrICVtk2OgwlIlIAheqifNGiRU3GL7744iaPWW1w7bXXAsGls3f95z1NuijPrqVQFBYicmjqgucqujIdhhIRkUgKCxERiaSwEBGRSAoLERGJpLAQkUPSNb+9hmt+e01nl9FtKCxERAqgoYvy5cuX8+lPf5qxY8dy7LHH8thjj+3XthBdlAMsW7aM559bWJD6o+jSWRGRAurTpw8//vGPGTNmDOvXr+f444/nvPPOY8CAAY1t8u2iPMqyZctYtGQRp31mckFqb02sexZmNsXM3jCzNWZ2U475V5vZZjNbHr6uzZr3FTN7K3x9Jc46RUQK5aijjmLMmDEAHHHEEQwZMoTNmzfnvfxbb73Feeedx/HHH8/kyZN58803AZgzZw7jxo1jwoQJnHnmmezdu5dZs2Yx98lKpp3xhTbvlbRVbHsWZpYE7gPOAaqAJWZW6e6vNWv6mLvPaLbsIOAHQAXgwMvhsh/GVa+IHBzufOlOXt/2emS7hjb5nLf45KBP8t0Tv9vmWl566SVqa2s58sgj815m+vTpzJ49myOPPJIXXniBGTNm8Mwzz3DbbbexYMECysrK2L59O71792bmzJksXLKIm//5+03u4I5DnIehTgTWuPtaADObA1wANA+LXM4Dfufu28JlfwdMAX4eU60iIgX1wQcf8OUvf5lHH32URCK/gzjbt2/nxRdfbNK9RzqdBuCUU07hqquu4tJLL+Wiiy6KpebWxBkWw4D3s8argJNytLvYzCYDbwI3uvv7LSw7rPmCZjYdmA4wcuTIApUtIt1ZvnsADXsUj0x5pOA17Ny5k8997nP80z/9E5MmTcp7OXenpKQk5zmMhx56iMWLFzN37lwmTJjAq6++WsiSI8V5zsJyTGveyfqvgHJ3PxZ4FmjofzefZXH3B929wt0rSktLD6hYEZFCqK2tZdq0aY17AW0xcOBAhg4d2vhY1EwmwyuvvALA2rVrmTRpErfffjsDBw5k3bp1FBcXs3vX7oJ/h1ziDIsqYETW+HBgfXYDd9/q7jXh6EPA8fkuKyLSFT3++OMsXLiQ//7v/268JLYtVzvNmTOH+++/nwkTJjB27Fjmzp0LwI033sj48eMZP348Z599NuPGjeOss87ijVWvc9GZU7vvCW5gCTDGzEYB64ArgC9mNzCzoe7+QTg6FVgdDs8HfmhmA8Pxc4GbY6xVROSANHQLfuWVV3LllVfmtUyuLspHjx6d8/kXlZWV+00rLS3lF88GeyHd9gS3u6fNbAbBD38SeNjdV5nZLGCpu1cCN5jZVCANbAOuDpfdZma3EwQOwKyGk90iIoUQx7mKg1msN+W5+zxgXrNpM7OGb6aFPQZ3fxh4OM76REQkP+ruQ0REIiksREQkksJCREQiKSxE5JD07pev4t0vX9XZZXQbCgsRkQLo6C7Kn3rqKe6+++6C1R9FXZSLiBRQIbsoT6fTpFK5f6anTZsGwOotawr8DXJTWIiIFNBRRx3VOJzdRXl2WLTm1FNP5fTTT+f555/noosuYtSoUfzwhz+ktraW0tJSfvrTnzJkyBBmz57NypUr+dr3Z/Cdv/8mo48oZ8mSJWzYsIF77rmnMUwKRWEhIgeVDT/8ITWro7sor349aJPPeYueR3+Sw2+5pc21tKeLcgg6Ily4MHgC3ocffsjUqVMxM+6//37uuece7rzzzv2W2bRpEy+88AIrVqzgsssuU1iIiHQH7emivMEVV1zROPzee+9x2WWXsWHDBmpqaprsuWS78MILMTOOPfZY1q1bd0C156KwEJGDSr57AA17FB/7yY8LXkN7uyhv0Ldv38bh66+/nltuuYXzzz+fZ599ljvuuCPnMj179mwcdt+vk+4DpquhREQK6EC6KM9lx44dDBs2DHfn0UcfjV4gJgoLEZECOtAuypu79dZbmTZtGqeffjplZWUFrLRtdBhKRKQACtVF+aJFi5qMX3zxxU0es9rg2muvBYJLZ+/6z3uadFHeUEshKSxE5JAUx7mKg5kOQ4mISCSFhYgcFOK4AuhgcqDbR2EhIt1er1692Lp1qwKjBe7O1q1b6dWrV7s/Q+csRKTbGz58OFVVVWzevLmzS+lwH+wKv/Pmulbb9erVi+HDh7d7PbGGhZlNAf6d4Bncs909590kZnYJ8AvgBHdfamY9gAeACiADfN3dF8RZq4h0X0VFRYwaNaqzy+gUVz/yfQAWX/NkrOuJLSzMLAncB5wDVAFLzKzS3V9r1q4YuAFYnDX57wDcfbyZDQF+Y2YnuHsmrnpFRKRlcZ6zOBFY4+5r3b0WmANckKPd7cBdQHXWtGOA5wDcfROwnWAvQ0REOkGcYTEMeD9rvCqc1sjMjgNGuPvcZsu+AlxgZikzGwUcD4xovgIzm25mS81s6aF4rFJEpKPEec7CckxrvFTBzBLAj4Crc7R7GDgaWAq8C/wJSO/3Ye4PAg8CVFRU6DIIEZGYxBkWVTTdGxgOrM8aLwbGAQvMDOBwoNLMprr7UuDGhoZm9ifgrRhrFRGRVsR5GGoJMMbMRoVXN10BVDbMdPcd7l7i7uXuXg68CEwNr4bqY2Z9AczsHCDd/MS4iIh0nNj2LNw9bWYzgPkEl84+7O6rzGwWsNTdK1tZfAgw38wywDrgy3HVKSIi0WK9z8Ld5wHzmk2b2ULbM7KG3wE+EWdtIiKSP3X3ISIikRQWIiISSWEhIiKRFBYiIhJJYSEiIpEUFiIiEklhISIikRQWIiISSWEhIiKRFBYiIhJJYSEiIpEUFiIiEklhISIikRQWIiISSWEhIiKRFBYiIhKpzWFhZgkz6x9HMSIi0jXlFRZm9j9m1j98LvZrwBtm9u14SxMRka4i3z2LY9x9J3AhwWNSR5LHc7HNbIqZvWFma8zsplbaXWJmbmYV4XiRmT1qZivMbLWZ3ZxnnSIiEoN8w6LIzIoIwuJpd68DvLUFzCwJ3Ad8FjgG+BszOyZHu2LgBmBx1uRLgZ7uPh44HviamZXnWauIiBRYvmHxAPAO0BdYaGYfA3ZGLHMisMbd17p7LTAHuCBHu9uBu4DqrGkO9DWzFNAbqM1jfSIiEpO8wsLd/8Pdh7n7+R54FzgzYrFhwPtZ41XhtEZmdhwwwt3nNlv2CWA38AHwHvCv7r4tn1pFRKTw8j3B/fXwBLeZ2X+Z2TLgrKjFckxrPHRlZgngR8A3c7Q7EagHjgBGAd80s9E56ppuZkvNbOnmzZvz+SoiItIO+R6G+tvwBPe5QClwDXBHxDJVwIis8eHA+qzxYmAcsMDM3gEmAZXhSe4vAr919zp33wS8AFQ0X4G7P+juFe5eUVpamudXERGRtso3LBr2Es4HHnH3V8i955BtCTDGzEaZWQ/gCqCyYaa773D3Encvd/dy4EVgqrsvJTj0dFa4J9OXIEhez/tbtUEm49RVD6a+ri+16UwcqxAR6fZSebZ72cyeITgkdHN4BVOrv6zunjazGcB8IAk87O6rzGwWsNTdK1tZ/D7gEWAlQSg94u6v5llrm3y4p5Zt7wbn3Y/6/m84rHcRJf16UNKvJyXFPSnt15PS4p77poXTS/r1oGcqGUdJIiJdTr5h8VVgIrDW3feY2WCCQ1Gtcvd5BPdlZE+b2ULbM7KGdxFcPhu7vj1THDbsd2TSvZl+7N+yZVcNm3fVsOWjWlav38nCXTV8VJ3OuWz/XqkwOFoOlVIFi4gcBPIKC3fPmNlw4ItmBvBHd/9VrJV1kF5FST7RcwH0hH/8zL/kbFNdV8+WXTVs2VXLlo9qgkAJ37fsqmXzrhpWb9jJwrdaDpbiXilKwxBpEirh3ktJ1rReRQoWEela8goLM7sDOAH4WTjpBjM72d0PiTurexUlGT6wD8MH9olsW11Xz9bdLYfKlo+CYNnyUQ07WwqWnqkwUHpSUrxvT6VxWhgqpcUKFhHpGPkehjofmOjuGQAzexT4C9Dtw6I6Xc22RAYDnnrrKVKJFEWJopzv+cwrShRxxGG9GDagd+S6a9L1bN1Vu3+ofFTTGCxvbPiIF3ZtZcfeupyfUdwz1WSvZL9Qadhz6deT3j0ULCLSPvmGBcAAoOHGuMNiqKVT7Krbxcai4PaPmX/KeTqlXVKWO1xyBk6yqGn7/imKBxYx0FIcHU4zS5JOJ6hNG7V1RnUd7K2FvTWwp8bZWQPrtzsfbciwpwbck5D1ck/Sp6gHA3r3YmCf3gzu05vB/fpQ0rc3pcV9KO3Xh7Livgzt35chxb07LFjcHcep93rcg/eMZ5q8WpuX3SbjGTJkyGTC92Zt0vX11Hk99fX11NanSWcy1GXqqQ/f0/X1pDMNr0zje30mHb4H8+o9E4x7PfUN4/X7amiYVh+ud3d1Bgwuf+I7Tb87jnvTKU2HGuZ7k7neMOTNxrOX9ObTyVpX089r0tJbmZf12c17+8n+7KbVNGuHZy26/2cDjYdyz/rJV8lb0w2Z3yJtXqK92ramtn6VXdVp8F5tW6gd8g2LfwH+YmZ/ILg6aTIHwV4FwKBegxhTE1xBfN8Xf0s6k6YuU0ddpq5xuLX3fOZlT29puT31e/L+zHqv3/+LFIWv4qB/lJbsCF/vOPBR+NrQtI27gScxkiQsRdJSpKwh2FLs8e0AnP3YF8h4PY6T8XoyHr6Twd3JUB+GQQYPf8jdM8GPJBmcDB35Jxs3dwMM3AiuSg+Hexp4glU79t1m1PRb574K3VqZ1/qV69bCcGtj+X5e9pTm09vfLpgatk4FYbGtdk8rNXUvUfcZHJBUHaRjXUOwmnwaufvPzWwBwXkLA77r7htaX6p7SFiCVPifcmi/oZ1cTX4ynmkxUFoNs/o60r7vPZ1Js7eulp3V1eyo3suO6hp2Vlezq6aG3bW17K6tYU9dLXvraqhO11FbXwdWD+HjTKp2N/wgZv0wksD3mxb+eLphliBBgoQlMBIkElnjliBpwXCCJAkzkongPWFJEpYgaUmSiX3tkokEyUSKZLhsKpEMhhMJkolkMJ5IkAqXS4XTUuFwUTKZNa1hPEFRIkkykaRHct97KpGgKJmkKFlEUTJoE4wn6ZFIkUolSCWMZMKy3hOc/pMrwJw/feVxwgtEGn88zGicJoGTHrkYgMXXPNnJlXQPJz1ycduOEbVTq6sws081m1QVvh9hZke4+7J4ypLWJCxBj2QPeiR7dOh66+ozbNtdy2d/9o+A8ZML7iaVSJBMGEXJfT+OjT+WyaY/molD9IfREsEtSamkHkwp3VdUHt3Tyjwnun8oOYgUJROU9e9FUa/g1NXYIw6aU1ciEqHVsHD3qJ5lRUTkEJDvfRYX5Zi8A1gRdvQnIiIHsbZ09/Fp4A/h+BkEHf8dZWaz3P0nMdQmIiJdRL5hkQGOdveNAGZWBvwncBKwEFBYiIgcxPINi/KGoAhtAo5y921mlvvW4m7k6sdrg4HpnVuHiEhXlW9YPG9mc4FfhOOXEDyLuy+wPZbKRESky8g3LK4HLgJOJbif6FHgSXd3op/FLSIi3Vy+d3C7mS0Cagnur3gpDAoRETkE5HVLqZldBrxEcPjpMmCxmV0SZ2EiItJ15HsY6nvACQ33VJhZKfAs8ERchYmISNeRb2c1iWY3323NZ1kzm2Jmb5jZGjO7qZV2l5iZm1lFOP4lM1ue9cqY2cQ8axURkQLLd8/it2Y2H/h5OH45zZ6t3ZyZJYH7gHMIOiBcYmaV7v5as3bFwA3A4oZp7v4zwqfymdl44Gl3X55nrRKzH/xsdTAQ+RR2AW2vttL2apuO2l557Vm4+7eBB4FjgQnAg+7+3YjFTgTWuPtad68F5gAX5Gh3O3AXUN3C5/wN+0JKREQ6Qd69oLv7k0BbOpgfBryfNV5FcMd3IzM7Dhjh7nPN7FstfM7l5A6ZwsjUc+yYD/nwox6w5S0Y/PHgIQMiItIo6nkWH5H7UWZGcEVt/9YWzzGt8bPMLAH8CLi6lfWfBOxx95UtzJ9OeN/1yJEjWymlFTvXUzKgmuFle+DeCuhXBuWnQvlpwWvwkQoPETnkRXVRXnwAn10FjMgaHw6szxovBsYBC8IH4hwOVJrZVHdfGra5glYOQbn7gwSHx6ioqGjffR8DRvD7JYfTp1c9Z9x2C7yzCP76PKwMd6L6HR6Gx6kwajIMGq3wEJFDTpwP41sCjDGzUcA6gh/+LzbMdPcdQEnDePjY1m81BEW453EpwfO+Y2bsqU7B8VcHL3fYthb+ujAIj3cWwcrwKuHiofvCo/w0hYeIHBJiCwt3T5vZDGA+kAQedvdVZjYLWOrulREfMRmocve1cdXYIrPg8NPgI6HimiA8tr4N7zwfvP66EFaE3WQVH5EVHqcqPETkoBTrY77dfR7NLrF195kttD2j2fgCYFJctbWJGZR8PHg1hseaMDwWwdoFsOLxoG3/YU3DY+AohYeIdHuxhsVBywxKxgSvir8NwmPLW/vC4+0/wKuPBW0bw+O0MDzKFR4i0u0oLArBDEqPCl4nfDUMjzezwuP3WeExPOuE+Wkw4GMKDxHp8hQWcTCD0k8ErxOu3RceDSfM1zwLr84J2h42oukJ84Ef69zaRURyUFh0hOzwOPHvgvDY/Ma+E+ZvPQOvhFcIHzay2TkPhYeIdD6FRWcwgyGfDF6N4fF6eJnu8/DWfHjlf4K2A0buO99RfmowLiLSwQ75sHB3UmknnezEIsxgyNHB68S/g0ymaXi88RtY/rOgbWN4hAEyYETrny0iUgCHfFjUb9/OiA3B8JufPpnUkCGkyoaQGjKEoiFDSA0pC6YNGUJR2RCSgwZhyZiTJZGAsmOC10nTw/BY3UJ4fCwIjlFheBw2PN7aROSQdMiHRaJHDzYPhFQ9jDrvXNIbN5HetIma1a+T3rIlOESULZkkVVJCqqyM1JDSMFCyQ6WUorIyEv37Y4W6yimRgLKxweukr+0Lj7+G5zze+DUs/2nQdmB500t1FR4iUgAKi7592dU3+FEfeuutTeZ5Ok1661bSmzaR3riRuk1BkKQ3bQ7G332PvUuWUr9jx36faz17Nu6lFA0ZQqo0DJXskCkrI9G7dzuKzgqPSdcF4bHptX17Hqvnwl8awmNUs/AY1vb1icgh75APi9ZYKkVRWRlFZWUwfnyL7TLV1aQ3bw6DZBN1GzcGgRKGTPWq16jbtADfu3e/ZRPFxY2HuBoDJTtkhgwhVVqKFRW1XGgiAYePC16N4bFqX79Wq38Ff/lJ0HbQ6Kbh0f+IA91MInIIUFgUQKJXL3qMGEGPES2fbHZ3Mrt3k964MStUNjUOpzdtYveSl0hv3gJ1dfstnxw8OOs8Su5QSQ4ahCUSYXiMD16T/j4Ij40r94XHa0/Dsh8HH9wYHpOh/BSFh4jkpLDoIGZGsl8/kv360fPII1ts55kM9du3N4ZK46GvrGDZu2oV9Vu37n8+JZUiVVoaHuYqyxEo55IaeyWJvn2w7D2PVdnhceS+7tg/dgr0HxrjVhGR7kJh0cVYIkFq0CBSgwbB0Ue32M7r6hrPp9Q17q1sbgyZ2nf+yu6XXiKT63xK795BoJQOIVU2jFTpP5AamKHIN5La/TapPz9N6qVHSaQInhyYfdiq+PAYv72IdFUKi27KioooOvxwig4/nNZOkWf27m08n9L8XEp60yb2rlxBeuMmvDr7Eeh9gD4k+vakqE89qdQ8Ur1+Sap3PamSwXzysF3UZYxd984I+7Wy4L3h6q9EOA0LrgizxL552e0TiX3DzeYFjzMhbJM9L7H/OhvXbVjWuvfNC9eTyJre8FkJ9mtvLbbPrrnhncZ1WM66khyWrMOBmqXPtf0/9CGofzI4DFuz9PedXEn30D9ZR8bj719OYXGQS/TuTY+RI+nRymNn3Z3MRx+1eC6lbuNGajasI/3eh/BaLdAXgPdX68cvH4PCP7O1V87o5Eq6h8GN2+v6Tq6kexhMiqIB+5/nLDSFhQTnU/r3J9m/Pz0//vEW23kmQ/22bSy46DTM4eT/+/PwvInjmUww7BnAw+FgHplm0zzT+DR29/pguPly7kCm6XjD57njWcNN2meypje8h+vftwzN5jWsn6zl9n1W0+UyWfWStbxnfUbTz/rrk8H5oPJpV7brv8+h5p2ngsu+y6d9qZMr6R7eeepn7OmAjqsVFpI3SyRIlZRQ2yP4l9l74sROrqh7WPd40D39hOtu6+RKuod1TwRPoZxw3axOrqR7WPfEEx2ynkSHrEVERLq1WMPCzKaY2RtmtsbMbmql3SVm5mZWkTXtWDP7s5mtMrMVZtYrrjprrDc11o47qUVEDhGxHYYysyRwH3AOUAUsMbNKd3+tWbti4AZgcda0FPBT4Mvu/oqZDQbiP4MjIiI5xblncSKwxt3XunstMAe4IEe724G7gOxrN88FXnX3VwDcfau718dYq4iItCLOsBgGvJ81XhVOa2RmxwEj3H1us2WPAtzM5pvZMjP7Tox1iohIhDivhsp1MVdj/xQW3Pn0I+DqHO1SwKnACcAe4Dkze9ndm1zYb2bTgekAI1u5j0BERA5MnHsWVUB2z3rDgfVZ48XAOGCBmb0DTAIqw5PcVcAf3X2Lu+8B5gGfar4Cd3/Q3SvcvaK0tDSmryEiInGGxRJgjJmNMrMewBVAZcNMd9/h7iXuXu7u5cCLwFR3XwrMB441sz7hye7Tgdf2X4WIiHSE2MLC3dPADIIf/tXA4+6+ysxmmdnUiGU/BP6NIHCWA8vc/ddx1SoiIq2L9Q5ud59HcAgpe9pGU2SpAAAKvklEQVTMFtqe0Wz8pwSXz4qISCfTHdwiIhJJfUMBd3yxHIBpnVuGiEiXpT0LERGJpLAQEZFICgsREYmksBARkUgKCxERiaSwEBGRSAoLERGJpLAQEZFIuilP2uy2Lx0NwPmdXEd3oe3VNtpebdNR20t7FiIiEklhISIikRQWIiISSWEhIiKRFBYiIhJJYSEiIpEUFiIiEklhISIikWINCzObYmZvmNkaM7uplXaXmJmbWUU4Xm5me81sefi6P846RUSkdbHdwW1mSeA+4BygClhiZpXu/lqzdsXADcDiZh/xtrtPjKs+ERHJX5x7FicCa9x9rbvXAnOAC3K0ux24C6iOsRYRETkAcYbFMOD9rPGqcFojMzsOGOHuc3MsP8rM/mJmfzSz03KtwMymm9lSM1u6efPmghUuIiJNxRkWlmOaN840SwA/Ar6Zo90HwEh3Pw74P8D/mFn//T7M/UF3r3D3itLS0gKVLSIizcUZFlXAiKzx4cD6rPFiYBywwMzeASYBlWZW4e417r4VwN1fBt4GjoqxVhERaUWcYbEEGGNmo8ysB3AFUNkw0913uHuJu5e7eznwIjDV3ZeaWWl4ghwzGw2MAdbGWKuIiLQitquh3D1tZjOA+UASeNjdV5nZLGCpu1e2svhkYJaZpYF64Dp33xZXrSIi0rpYH37k7vOAec2mzWyh7RlZw08CT8ZZm4iI5E93cIuISCQ9VhU4Zuh+F1qJiEgW7VmIiEgkhYWIiERSWIiISCSFhYiIRFJYiIhIJF0NBTwy5ZHOLkFEpEvTnoWIiERSWIiISCSFhYiIRFJYiIhIJIWFiIhEUliIiEgkhYWIiERSWIiISCSFhYiIRFJYiIhIpFjDwsymmNkbZrbGzG5qpd0lZuZmVtFs+kgz22Vm34qzThERaV1sYWFmSeA+4LPAMcDfmNkxOdoVAzcAi3N8zI+A38RVo4iI5CfOjgRPBNa4+1oAM5sDXAC81qzd7cBdQJO9BzO7EFgL7I6xRmmHxdc82dkliEgHizMshgHvZ41XASdlNzCz44AR7j43+1CTmfUFvgucQ7MQabb8dGA6wMiRIwtXuUgBKVzbRturbTpqe8V5zsJyTPPGmWYJgsNM38zR7jbgR+6+q7UVuPuD7l7h7hWlpaUHVKyIiLQszj2LKmBE1vhwYH3WeDEwDlhgZgCHA5VmNpVgD+QSM7sLGABkzKza3e+NsV4REWlBnGGxBBhjZqOAdcAVwBcbZrr7DqCkYdzMFgDfcvelwGlZ028FdikoREQ6T2yHodw9DcwA5gOrgcfdfZWZzQr3HkREpJswd49u1Q1UVFT40qVLO7sMEZFuxcxedveKqHa6g1tERCIpLEREJJLCQkREIiksREQk0kFzgtvMNgPvHsBHlABbClROIamutlFdbaO62uZgrOtj7h55V/NBExYHysyW5nNFQEdTXW2jutpGdbXNoVyXDkOJiEgkhYWIiERSWOzzYGcX0ALV1Taqq21UV9scsnXpnIWIiETSnoWIiEQ6ZMPCzC41s1Vmlmn+7O9m7fJ6jngB6xpkZr8zs7fC94EttKs3s+XhqzLGelr9/mbW08weC+cvNrPyuGppQ01Xm9nmrO1zbdw1het92Mw2mdnKFuabmf1HWPerZvapLlLXGWa2I2t7zeygukaY2R/MbHX4t/j1HG06fJvlWVeHbzMz62VmL5nZK2Fdt+VoE9/fo7sfki/gaOATwAKgooU2SeBtYDTQA3gFOCbmuu4CbgqHbwLubKHdrg7YRpHfH/gH4P5w+ArgsS5Q09XAvZ3wb2oy8ClgZQvzzyd4prwBk4DFXaSuM4C5nbC9hgKfCoeLgTdz/Lfs8G2WZ10dvs3CbdAvHC4CFgOTmrWJ7e/xkN2zcPfV7v5GRLPG54i7ey3Q8BzxOF0APBoOPwpcGPP6WpPP98+u9wngMxY+zaoTa+oU7r4Q2NZKkwuAH3vgRWCAmQ3tAnV1Cnf/wN2XhcMfETzKYFizZh2+zfKsq8OF26Dh6aFF4av5SefY/h4P2bDIU67niMf9j6bM3T+A4B8tMKSFdr3MbKmZvWhmcQVKPt+/sY0HzzDZAQyOqZ58awK4ODxs8YSZjcgxvzN0xr+nfH06PLzxGzMb29ErDw+XHEfwf8vZOnWbtVIXdMI2M7OkmS0HNgG/c/cWt1eh/x7jfFJepzOzZwke19rc99z96Xw+Ise0A758rLW62vAxI919vZmNBn5vZivc/e0Dra2ZfL5/LNuoFfms71fAz929xsyuI/g/rbNirClfHb2t8rWMoMuHXWZ2PvBLYExHrdzM+gFPAt9w953NZ+dYpEO2WURdnbLN3L0emGhmA4CnzGycu2efi4ptex3UYeHuZx/gR0Q9R7xdWqvLzDaa2VB3/yDc3d7UwmesD9/XWvBI2uMIjuUXUj7fv6FNlZmlgMOI95BHZE3uvjVr9CHgzhjraYtY/j0dqOwfQnefZ2b/z8xK3D32PpDMrIjgB/ln7v6/OZp0yjaLqqszt1m4zu3h3/0UIDssYvt71GGo1jU+R9zMehCcMIrtyqNQJfCVcPgrwH57QGY20Mx6hsMlwCnAazHUks/3z673EuD3Hp5di0lkTc2OaU8lOObcFVQCV4VX+EwCdjQccuxMZnZ4w3FtMzuR4Hdha+tLFWS9BvwXsNrd/62FZh2+zfKpqzO2mZmVhnsUmFlv4Gzg9WbN4vt77Miz+V3pBUwjSOEaYCMwP5x+BDAvq935BFdDvE1w+CruugYDzwFvhe+DwukVwOxw+GRgBcGVQCuAr8ZYz37fH5gFTA2HewG/ANYALwGjO2AbRdX0L8CqcPv8AfhkB/2b+jnwAVAX/tv6KnAdcF0434D7wrpX0MJVeJ1Q14ys7fUicHIH1XUqwSGSV4Hl4ev8zt5medbV4dsMOBb4S1jXSmBmOL1D/h51B7eIiETSYSgREYmksBARkUgKCxERiaSwEBGRSAoLERGJpLAQaQMz2xXdqtXlnwjvusfM+pnZA2b2dtiL6EIzO8nMeoTDB/VNs9K9KCxEOkjYf1DS3deGk2YT3F07xt3HEvSWW+JBB4nPAZd3SqEiOSgsRNohvKP4bjNbaWYrzOzycHoi7PphlZnNNbN5ZnZJuNiXCO/IN7MjgZOA77t7BoKuW9z912HbX4btRboE7eaKtM9FwERgAlACLDGzhQRdr5QD4wl6DF4NPBwucwrB3dQAY4HlHnQMl8tK4IRYKhdpB+1ZiLTPqQQ929a7+0bgjwQ/7qcCv3D3jLtvIOhupMFQYHM+Hx6GSK2ZFRe4bpF2UViItE9LD5Rp7UEzewn67oGgX6EJZtba32BPoLodtYkUnMJCpH0WApeHD6MpJXh06UvAIoIHLyXMrIzg8ZsNVgMfB/Dg2SNLgduyei8dY2YXhMODgc3uXtdRX0ikNQoLkfZ5iqD3z1eA3wPfCQ87PUnQs+tK4AGCJ6ztCJf5NU3D41qCh2CtMbMVBM/eaHhWw5nAvHi/gkj+1OusSIGZWT8PnqA2mGBv4xR33xA+g+AP4XhLJ7YbPuN/gZs9+jnxIh1CV0OJFN7c8CE1PYDbwz0O3H2vmf2A4DnJ77W0cPhQp18qKKQr0Z6FiIhE0jkLERGJpLAQEZFICgsREYmksBARkUgKCxERiaSwEBGRSP8f0jmLuSrXG5QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8b9c6a8c88>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的logloss。\n",
    "可以看出在训练集上C越大（正则越少）的模型性能越好；\n",
    "但在测试集上当C=100时性能最好（L1正则）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 正确率参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='liblinear',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=4,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='accuracy',n_jobs = 4,)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7747395833333334\n",
      "{'C': 0.1, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VfWd//HXJ3uAYAKERcK+iLIkKAqKxV0ptSpaEVvr1JkO0452Wmem09b2Z106HZdax06dutBau0yp1dpSSgXBBbEuIA2CIvsW9i0gS/bP749zEm5CknsDublZ3s/H4z7uued8zzmfeyDnc7/f7znfY+6OiIhIY5ISHYCIiLR+ShYiIhKVkoWIiESlZCEiIlEpWYiISFRKFiIiEpWShYiIRKVkISIiUSlZiIhIVCmJDqC59OjRwwcOHJjoMERE2pT33ntvr7vnRivXbpLFwIEDWbp0aaLDEBFpU8xscyzl1AwlIiJRKVmIiEhUShYiIhJVu+mzEJGOq7y8nKKiIkpKShIdSquVkZFBXl4eqampJ7W+koWItHlFRUVkZWUxcOBAzCzR4bQ67s6+ffsoKipi0KBBJ7UNNUOJSJtXUlJC9+7dlSgaYGZ07979lGpeShYi0i4oUTTuVI+PkoWIdEg3PfkWNz35VqLDaDOULIDbXrqN2166LdFhiEgb1qVLl5rpyZMnk52dzdVXX11v2dtvv52CggLOOussMjMzKSgooKCggOeff75J+1y2bBkvvfTSKcUdK3Vwi4g0s69//escPXqUJ598st7ljz/+OACbNm3i6quvprCw8KT2s2zZMlauXMnkyZNPOtZYqWYhItLMLrvsMrKysk5q3bVr13LVVVdxzjnnMGnSJNasWQPArFmzGDVqFPn5+VxyySUcO3aM++67j1//+tcnVStpKtUsRKRdufdPH/Dh9kNRy324IygTS7/FWad35bufHnnKscVixowZzJw5kyFDhvDmm29yxx13MH/+fO69915ee+01evXqRXFxMZmZmdx9992sXLmS//7v/457XEoWIiKtRHFxMW+//TY33HBDzbyKigoAJk6cyK233sqNN97I9ddf3+KxKVmISLsSaw2gukbx2386P57hNIm706NHj3r7MJ5++mneeecd5syZQ35+Pu+//36LxqY+CxGRViInJ4c+ffrw4osvAlBVVcXy5csB2LBhAxMmTOD+++8nJyeHbdu2kZWVxccff9wisSlZiIg0s0984hPceOONLFy4kLy8PObNmxfzurNmzeKJJ54gPz+fkSNHMmfOHADuvPNORo8ezejRo7n88ssZNWoUl156KcuXL2fs2LHq4BYRaQsOHz5cM/3GG2/EtM7AgQNZuXJlrXmDBw+uN7nMnj37hHm5ubkt9tA3JQsR6ZBaU19FWxDXZigzm2xmq81snZl9s57lj5pZYfhaY2bFEcseNLOV4eumeMYpIiKNi1vNwsySgceBK4AiYImZzXb3D6vLuPudEeW/AowNpz8FnA0UAOnA62b2F3ePfvG0xF310CjPTH4mwZG0DeOfCS6DfOe2FxIcSdug49U0q/auA+DMHkPjup941izOA9a5+wZ3LwNmAdc2Uv5m4Dfh9FnA6+5e4e5HgOVA/O9nFxGResUzWfQFtkZ8LgrnncDMBgCDgFfCWcuBT5pZJzPrAVwC9ItjrNIEH+44VHP3q4h0DPHs4K5v8HRvoOx04Hl3rwRw9/lmdi7wV2AP8BZQccIOzGYAMwD69+/fHDGLSEfxzKeC99v+nNg42oh41iyKqF0byAO2N1B2OseboABw9/909wJ3v4Ig8aytu5K7P+Xu49x9XG5ubjOFLSLSdC09RPmLL77Iww8/fMpxxyqeNYslwDAzGwRsI0gIn61byMzOAHIIag/V85KBbHffZ2ZjgDHA/DjGKiLSbJpriPKKigpSUuo/TU+dOhU43sEdb3GrWbh7BXAHMA9YBTzn7h+Y2X1mdk1E0ZuBWe4e2USVCrxhZh8CTwG3hNsTEWn1TmWI8gsvvJBvf/vbTJo0iR//+Mf88Y9/ZPz48YwdO5Yrr7yS3bt3AzBz5ky+9rWvAfAfX/43vvrVr3LBBRcwePDgmuFCmlNcb8pz97nA3Drz7q7z+Z561ishuCJKRKRp/vJN2Lkiermd4UB81X0Xjek9Gj75wKnF1QSHDh1i0aJFABw4cIBrrrkGM+OJJ57gkUce4cEHHzxhnd27d/Pmm2+yYsUKpk2bVlPzaC66g1tEpJWZPn16zfSWLVuYNm0aO3fupLS0lOHDh9e7znXXXYeZMWbMGLZt29bsMSlZiEj7EmsNoBVfDdW5c+ea6dtvv5277rqLKVOmsGDBAh54oP7vl56eXjNdu1W/eWjUWRGRVuzgwYP07dsXd+fZZ59NWBxKFiIizexUhiiv65577mHq1KlcdNFF9OrVqxmjbBo1Q4mINIPmGqJ88eLFtT7fcMMNtR6zWu2LX/wiEFw6+9BPHqk1NlRkLM1FyUJEOqZW2FfRmqkZSkREolKyEBGRqJQsREQkKiULERGJSslCRDqk2166reapjxKdkoWISDOoHqK8sLCQ888/n5EjRzJmzBh++9vfnlC2OYYoB1i2bBlvLFzULPFHo0tnRUSaUadOnfjFL37BsGHD2L59O+eccw5XXXUV2dnZNWViHaI8mmXLlrF4yWI+cdmkZom9MapZiIg0o+HDhzNs2DAATj/9dHr27MmePXtiXn/t2rVcddVVnHPOOUyaNIk1a9YAMGvWLEaNGkV+fj6XXHIJx44d47777mPOC7OZevGnm1wraSrVLESkXXnw3Qf5aP9HUctVl4ml32JEtxF847xvNDmWd999l7KyMoYMGRLzOjNmzGDmzJkMGTKEN998kzvuuIP58+dz77338tprr9GrVy+Ki4vJzMzk7rvvZtGSxXzrP79T6w7ueFCyEBGJgx07dvD5z3+eZ599lqSk2BpxiouLefvtt2sN71FRETz3beLEidx6663ceOONXH/99XGJuTFKFiLSrsRaA6iuUTwz+Zlmj+HQoUN86lOf4nvf+x4TJkyIeT13p0ePHvX2YTz99NO88847zJkzh/z8fN5///3mDDkq9VmIiDSjsrIypk6dWlMLaIqcnBz69OlT81jUqqoqli9fDsCGDRuYMGEC999/Pzk5OWzbto2srCyOHD7S7N+hPkoWIiLN6LnnnmPRokX8/Oc/r7kktilXO82aNYsnnniC/Px8Ro4cyZw5cwC48847GT16NKNHj+byyy9n1KhRXHrppaz+4COuv+QadXCLiLQF1cOC33LLLdxyyy0xrVPfEOWDBw+u9/kXs2fPPmFebm4uv1sQ1ELUwS0iEgfx6Ktoz9QMJSIiUSlZiIhIVEoWIiISlZKFiIhEpWQhIh3S5s/fyubP35roMNoMJQsRkWbQ0kOUv/jiizz88MPNFn80unRWRKQZNecQ5RUVFaSk1H+anjp1KgCr9q5r5m9QPyULEZFmNHz48JrpyCHKI5NFYy688EIuuugi3njjDa6//noGDRrE97//fcrKysjNzeVXv/oVPXv2ZObMmaxcuZJ/+s4d/MeX/43Bpw9kyZIl7Ny5k0ceeaQmmTQXJQsRaVd2fv/7lK6KPkR5yUdBmVj6LdLPHEHvu+5qciwnM0Q5BAMRLloUPAHvwIEDXHPNNZgZTzzxBI888ggPPvjgCevs3r2bN998kxUrVjBt2jQlCxGRtuBkhiivNn369JrpLVu2MG3aNHbu3ElpaWmtmkuk6667DjNjzJgxbNu27ZRir4+ShYi0K7HWAKprFAN++Ytmj+Fkhyiv1rlz55rp22+/nbvuuospU6awYMECHnjggXrXSU9Pr5l296YHHYWuhhIRaUanMkR5fQ4ePEjfvn1xd5599tlmiPDkKFmIiDSjUx2ivK577rmHqVOnctFFF9GrV69mjLRp1AwlItIMmmuI8sWLF9f6fMMNN9R6zGq1L37xi0Bw6exDP3mk1hDl1bE0p7gmCzObDDwGJAMz3f2BOssfBS4JP3YCerp7drjsIeBTBLWfl4Gvejwa4kSkQ4pHX0V7FrdkYWbJwOPAFUARsMTMZrv7h9Vl3P3OiPJfAcaG0xcAE4Ex4eLFwEXAa/GKV0REGhbPPovzgHXuvsHdy4BZwLWNlL8Z+E047UAGkAakA6nArjjGKiJtnBoeGneqxyeeyaIvsDXic1E47wRmNgAYBLwC4O5vAa8CO8LXPHdfFcdYRaQNy8jIYN++fUoYDXB39u3bR0ZGxklvI559FlbPvIb+JacDz7t7JYCZDQXOBPLC5S+b2SR3X1RrB2YzgBkA/fv3b5agJboB5esTHYJILXl5eRQVFbFnz55Eh9LidhwOv/Oe8kbLZWRkkJeX12iZxsQzWRQB/SI+5wHbGyg7Hbg94vNU4G13PwxgZn8BJgC1koW7PwU8BTBu3LiT/kkx/X8+CCYmn+wWRCSRUlNTGTRoUKLDSIgvPPMdAN657YW47ieezVBLgGFmNsjM0ggSwuy6hczsDCAHeCti9hbgIjNLMbNUgs5tNUOJiCRI3JKFu1cAdwDzCE70z7n7B2Z2n5ldE1H0ZmBWnctinwfWAyuA5cByd/9TvGIVEZHGxfU+C3efC8ytM+/uOp/vqWe9SuCf4hmbiIjETndwi8SZLgiQ9kDJQiTOvvBcWTAxI7FxtBXf/XXYPXlbYuNoK1rqeGkgQRERiUo1C2ky/VIW6XiULICjZZWJDkFEpFVTM5SIiESlZCEiIlEpWYiISFRKFiIiEpWShYiIRKVkISIiUenSWSDdjyU6BBGRVk01CxERiUrJQkREolKyEBGRqJQsREQkKiULERGJqsNfDVVeVc7PLk0ib5/TddtixuSOoWta10SH1Tod2QcbXmX00AOkplTBrM8lOqI24ewR+3CA5/8eklIhOQWSUsLp1GA6OTX4nJQSLq+7LCVi3eplyRHTEcsi16m1rM66Ik3Q4ZPFvmP7+DgDXhllLFzwZQxjSPYQ8nPzKehZQH5uPgO7DsTMEh1qy6uqhG3vwboFwWvbMsDp1T2JkrIk2L8x0RG2Kl7leAVUVThVFcenk48a5nBkyXvglcFxraoIpisrCQuCV7VgtBYmm2SwlHC6OqEkgyUfn45cZkm1y51Qtrpc5HpN20bPlDLc4cjsn7bg8Wi7cpPLqGiB/Zi7t8Bu4m/cuHG+dOnSk1p37uVnUZoCpz/zDMv3LKdwdyHL9yznUNkhALLTs2slj1E9RpGZktmc4bceH++EdQuD5LD+FSgpDv64886FoZfD0MuY+3e3AMaUBR8mOtqo3B0vK8NLSqgqKcGPHaOqtDR4r55XUkLVsRK8NHwvOUZVSWnwHjG/qrQEPxaxTp3teXl5or+udFCp2eUMfXvdSa1rZu+5+7ho5Tp8zaJaegWM7zOe8X3GA1DlVWw6uInCPYU1yeP1otcBSLZkzuh2BgW5BRT0LKAgt4DenXu3zdpHZTlsfed47WHnimB+l94w4moYehkMvhg6dYtY6dS/p1dUHD8h13diLjmGl5QG75En6vpO2OE2TigbluFkfhAlJ5OUkYFlZpKUno5lZpCUkYllpJPctStJvXpi6RkkZWZgGZkkZaRjGcfLJGVkhssyWHL37QCM/8HPT/m4tT1euyZVFVGzqqld1V724ePfA4OzvvSNRAffJnz4xINUOAyN836anCzMLAno4u6H4hBPq5FkSQzOHszg7MFcP+x6AA6WHqxV83hx3Yv830f/B0DPTj0pyC2oqYGc2e1MUpNTE/kVGla89Xhy2PA6lH0MSSl43nh80nfwvAupyhoYnGxLS6lavRkvWV1z8u1yxDGHfT97JvrJvdZJvaTm1zwn+SvcMjLCk3dmcDLPyAhP6hmkdu0anNTTM2qd3GtO3DUn94zj62ZmYunpJNXdXmrz/duVfD9Irp0njG+2bbZne370EACdr/vHBEfSNuz58aMtsp+YkoWZ/R/wJaASeA84zcx+6O4PxzO41ua09NOYlDeJSXmTAKioqmDNgTU1yWP5nuXM3zwfgPTkdEZ2H0l+z/yaJNI9s/tJ7dfLy2s1ndScdI8dC07mke8lJSf+Uj9yhKr9RfiB7VQV7wmWVRjuqVRZLl7Zi6rySrxkI7AR+Fmj8eSG77sfCv6oSUmpdaKtOTFnZJCcnR3+6j7x5B55Uj9eJnyv7+Seno4l6QI+kUSItWZxlrsfMrPPAXOBbxAkjQ6VLCJ5VRVJpeUMT+rDsNOyuSH9fLzHMfYV72D9rlVs2r2arZvXseXAz9leVsmCCsi1ruSl5tI7pRs9rCtZngalpWE7efWv7upf6Mfbw6k4ue4rS0shKcUxyklKrsRSjKTOXUjqeTrJOb1J6toj/IVe50SdkRH8Im/gV/fr/3gNbnD5n5YEv/Kb8Ve4iLROsSaLVDNLBa4Dfuzu5WbWLnrGKw8fIedg0Kyy4557TjhR1/01X/NeWtrgNnPD17knLCmmPLmY0lTYnwK7UpNIyswgLTOLTl2y6ZLTnfTM0+v8qm7g13bdk3iyY3vfJ6nor9jW17GDGzEDug2GoVOCzumBEyGt8ykfs4qUoFkluUuXU96WiLQNsSaLJ4FNwHJgkZkNANpFn4WXl3Hax+AGH7+8oHZTSUYGSV26kJzb43iTSdSTeO2mk7rbIymJosNFtZqu1hxYQ5Xvw9hQ67LdgtwCBnQdUH/HuTvsWR30O7y/ADb/FSpLIbUTDJoEF94OQy6F7kNa/qCKSLsTU7Jw9x8BP4qYtdnMLolPSC0rOTubTX0BM6YsWNwi++yX1Y9+Wf349JBPA3Ck/Agr9q6gcHchhXsKmb95Pi+sfQEILtstyC0I+j5OG87IQ3vI3LgouLz14NZgg7kj4Lx/DGoP/c+H1IwW+R4i0nHE2sH9VeAZ4GNgJjAW+CYwP36htQwzgwRf8to5tTMT+kxgQp8JQHDZ7saDGync/TcKN79G4a6/8VrRawCkuHNGeSUFub0pyL+agjOn0bvP2QmMXkQ6glibof7e3R8zs6sImuNvI0gebT5ZtDpH95O04VWGrFvIkHULuOHwLgCKe4/k/bwxFHbOorB0L7/f9wG/3vpn2Prnmst2q5uuRnQbEdfLdvel5VCWlMrew6VkZ6aSkqwrlETau1iTRfVP7ynAM+6+3NrkHWitUFUlbC+MGFJjaTDsQ0Z20Ocw7AoYcinZWb2ZBEwKVyuvKj9+2e7u+F22G2nj3iP88OU1/GnMXQDc/70FAGRlpJDTKY2czmnkdEoNpjsF09nhvG6d0sjulEZO52B5RqrGJhJpS2JNFu+Z2XxgEPAtM8sCWnIgm/bl8O5gKI11C4K+h2P7AYO+Z8Ok/wj6Hvqe3ehgb6lJqYzsPpKR3UfyuTODAf12HdkV3DS4J0ggv/zwlzxT9QwA/bP61wxXkp+bz9DsoSTHOJjczoMlPLZwLc8t3UpachJX7VhI36M7GPiv/8X+I2UUHy3nwNEyDhwtZ9/hMtbtPkzx0XIOlzZ8yW9manKQTDql0a1zGtk1SSY1TDrH51Uv75Ke0jbvkhdpB2JNFv8AFAAb3P2omXUnaIqSWFRWQNGSMDm8DDuWB/M758Lwq4LkMPgS6Hxqv/57de7FlZ2v5MqBVwJQUlHCqv2rgo7z3YUs3raY2etnB7tO7cyYHmNqmq5G544mKy2r1vYOHCnjJ6+v59m/bqLKnVvG9+f2S4eydOq/AjDl/IGNxlNWUUVxmEQOHC3jwJETp4PlZWwrPsaBo2UcPFbe4OgcqckW1E7CJJPTKTVMJMfndQtrL9XTXTNTSU5SghE5VbFeDVVlZnnAZ8Nfdq+7+5/iGllbd3AbrF8Ia18OhtQoPRiMrNlvPFz6/4IE0XsMxPGO5IyUDMb2HMvYnmOBYFC9oo+Lao139eT7T1LlVTWj7Rb0LGBEzmhWb+rOc28d40hZJVPH9uXOy4fTr1unJu0/LSWJnl0z6Nk19quzKqucQ8fK2X+0LEgkRyKmw+SyP0w0G/ceYdmWYg4cKaOiqv4MYwanZUbUWjodTy7VNZi609md0khLUT+MSKRYr4Z6gOAes1+Hs/7FzC5w92/FLbK2pqIUtrwd1BzWLYTd4YisWafDyGuD5DDoIsjMTliIZka/rv3o1/X4ZbuHyw4Hl+3uKWTZrkJmr5vL81XPA5A6uAuX9CzgrLxz2FVWQY+KkXEfbTc5yYITd+e0mNdxdw6XVlB8tDxMJBFNYxG1meKj5ew4WMKqHYc4cLScY+WVDW6zS3pKTTNYdliDiZzOjkg+1X01manJaiaTdivWZqgpQIF7MOC+mT0L/A3o2MniwKag5rBuIWxcBOVHgofLDLgArrg/SBA9z0z4pbmN6ZLWhXN7jWfrtjw+eG84+w5ezdlDy7hw1BH2V6xh+Z7lPLYsuP8kxVIY0W0Ep40zOpfC1uVPJjj6RiQBXaBzF+gM5NVTpKLKOVZewbGyKkrKKzkWvkrKak9vLq/io32VlOyspLSRBJOclERmWjIZqUlkpiaTmZZMZmoyRy8YRhJV/O7F78fr27Yrey8IbiTV8YrN3guGkHOkiilx3k9TRp3NBvaH06fFsoKZTQYeA5KBme7+QJ3ljwLVN/d1Anq6e3Z4w1/kUIojgOnu/ocmxNv8yo/BpjfD2sMC2BeOH589AApuDofU+ASkt41hMNydv6zcySPzV7N+zxHy807joc/kc+GwHrXKHSg5UHO3eeHuQt4dZpSnGPMLf5ygyFtQEpAevo6/Nag8fH0MwSUgpUB4E/2mQ5viEWH7o+PVNEOg6HDPuO8m1mTxX8DfzOxVgstoJxGlVmFmycDjwBVAEbDEzGa7e80Tc9z9zojyXyG42Q93f5WgQx0z6wasIxH3dLgHCWFtmBw2vwkVJZCSESSFc8O7prsPadW1h7rcnTfW7uXheatZse0gQ3t24YlbzuaqkfU/kyMnI4eL+13Mxf0uBmDO5WfhBp+c934LR942/X7yeMC45s9/TXQobcJLU4KbTCfPXZbgSNqG4Hjtgtvju59YO7h/Y2avEfRbGPANd98ZZbXzgHXuvgHAzGYB1wINPV7tZuC79cz/DPAXdz8aS6ynrPTjoEmp+r6H4i3B/O7DYNzfBw8DGjARUtvmk/KWbTnAQy99xNsb9tM3O5Mf3JjP1LF9m3TFUBKAQ0qSnp0Vi1QHcDJSY++H6cjSwsfL6njFJq2FHsfb6F+7mdUdR6IofD/dzE5398ZSf19ga5116336Szgw4SDglXoWTwd+2Ficp8SdrE7l5OaUwM+vDjqpq8ohrUvQIT3xa0GCyBkYtxBawuqdH/PwvNUsWLWLHl3SuOfTZ3Hz+P6kp+jmuHh74LMDAZia2DDajHs/dyZA3Nvg24uWOl7Rfho+0sgyBy5tZHl9P1UbGtZ8OvC8u9fqPTSzPsBoYF69OzCbAcwA6N+/fyOhNKJ4M584e3cwfXQ/nP/PQdNSvwmQ0vZ/2WzZd5RHF6zhD4Xb6JKWwr9fOZzbJg6ic7pqBSISu0bPGO5+KiPLFgH9Ij7nAdsbKDud+lvcpgEvunu9z+B096eApwDGjRt3cs/XyBnI3z7KYf+hdC6b237alHcfKuF/XlnHrCVbSDJjxqTBfPmiIWR3avsJUERaXqz3WVxfz+yDwAp3393AakuAYWY2CNhGkBA+W8+2zwBygLfq2cbNtMDluTv2Nu1ms9bs4NFynli0nmfe3EhFpXPTuf34l8uG0asJN8ZFo2YCkY6nKcN9nA+8Gn6+GHgbGG5m97n7L+uu4O4VZnYHQRNSMvAzd//AzO4Dlrr77LDozcAs99qDPJjZQIKayetN+kYd1NGyCp55cxNPvr6ej0sruCb/dO68fDgDe5z6k/FERGJNFlXAme6+C8DMegE/IeiwXgSckCwA3H0uwTO7I+fdXefzPQ2su4mgk1waUVZRxawlW/jRwnXsPVzKZSN68u9XncGZfbomOjQRaUdiTRYDqxNFaDcw3N33m1m9/QkSX5VVzh8Lt/HogjVs3X+M8wZ144lbzmbcwG6JDk1E2qFYk8UbZjYH+F34+TMEz+LuDBTHJTKpl7vz8oe7+MH81azZdZiRp3fl57eN4qLhuRqXSETiJtZkcTtwPXAhwSWxzwIvhP0M7eJZ3G3BX9ft5aF5qyncWszgHp15/LNn88lRvUnSENwiEmex3sHtZrYYKCO4V+Lduh3SEj/Ltxbz8LzVLF63lz6nZfDgDaO54ew8Pc5URFpMrJfOTgMeBl4jqFn8j5l93d2fj2NsHd663R/zg3lreOmDnXTrnMZ3PnUmt0wYoEeSikiLi7UZ6tvAudX3VJhZLrAAULKIg6IDR/nvBWv5/bIiOqWl8LXLh/EPFw4iKyM10aGJSAcVa7JIqnPz3T7C8eSk+ew9XMqPX1nH/72zBQz+fuIg/vmSoXRrwoOARETiIdZk8ZKZzQN+E36+iTr3T8jJO1RSztOLNvDTxRsprajixnPy+JfLhnF6dtsc2VZE2p9YO7i/bmY3ABMJ+iyecvcX4xpZB1BSXsmzf93ET15fT/HRcj41pg//esVwhuS2jYcniUjHEfPQo+7+AvBCHGPpMMorq3hu6VZ+tHAtuw6VctHwXL5+1RmM6hvTAwhFRFpctOdZfEz9w4obwRW17WJMiZ9PC/oE4j0wXlWV86f3t/Poy2vYtO8o5wzI4UfTxzJ+cPc471lE5NREG6I8q6UCac/cnVdX7+bheWtYteMQI3pn8dO/G8elI3rqrmsRaRP0BJw4e3fjfh566SOWbj7AgO6deGx6AZ8ec7ruuhaRNkXJIk5WbjvID+av5rXVe+iZlc5/Th3FtHH9SNVd1yLSBilZNLMNew7zw5fXMOf9HZyWmcq3PjmCW88fSGaa7roWkbZLyaKZ7Dh4jMcWrOV37xWRnpLEVy4dyj9OGkxX3XUtIu2AksUp2n+kjP99dR2/eHszOHx+wgBuv2QouVnpiQ5NRKTZKFmcpMOlFcx8YwMz39jI0bIKrj87j69dPoy8nPbzPG8RkWpKFk1UUl7Jr97ezP++tp79R8qYPLI3/3blcIb10lXGItJ+KVnEqKKyiheWFfHYgrWHyWvmAAAML0lEQVRsP1jChUN78PWrziC/X3aiQxMRiTsliyiqqpy/rNzJIy+vZsOeI+T3y+YHN+ZzwdAeiQ5NRKTFKFk0wN1ZtHYvD8/7iJXbDjGsZxee/Pw5XHlWL911LSIdjpJFPd7bfICHXvqIdzbuJy8nk0duzOe6sX1J1l3XItJBKVlE+GjnIX4wbzULVu2mR5d07rt2JNPP7U9aiu66FpGOTckCWO8FHNk7lk8+9gZd0lP4+lVncNvEgXRK0+EREQElCzbuPcK+jTeAVfHli4bwpUlDOK2T7roWEYnU4ZPFoB6dyer5LulZG/nG5F8lOhwRkVapwycLgE45HyY6BBGRVk09tyIiEpWShYiIRKVkISIiUSlZiIhIVEoWIiISlZKFiIhEpWQhIiJRxTVZmNlkM1ttZuvM7Jv1LH/UzArD1xozK45Y1t/M5pvZKjP70MwGxjNWERFpWNxuyjOzZOBx4AqgCFhiZrPdveYOOHe/M6L8V4CxEZv4BfCf7v6ymXUBquIVq4iINC6eNYvzgHXuvsHdy4BZwLWNlL8Z+A2AmZ0FpLj7ywDuftjdj8YxVhERaUQ8k0VfYGvE56Jw3gnMbAAwCHglnDUcKDaz35vZ38zs4bCmIiIiCRDPZFHfk4K8gbLTgefdvTL8nAJ8Avh34FxgMPCFE3ZgNsPMlprZ0j179px6xCIiUq94JosioF/E5zxgewNlpxM2QUWs+7ewCasC+ANwdt2V3P0pdx/n7uNyc3ObKWwREakrnsliCTDMzAaZWRpBQphdt5CZnQHkAG/VWTfHzKozwKWAhoYVEUmQuCWLsEZwBzAPWAU85+4fmNl9ZnZNRNGbgVnu7hHrVhI0QS00sxUETVpPxytWERFpXFyfZ+Huc4G5debdXefzPQ2s+zIwJm7BiYhIzHQHt4iIRKVkISIiUSlZiIhIVEoWIiISlZKFiIhEpWQhIiJRKVmIiEhUShYiIhKVkoWIiESlZCEiIlEpWYiISFRxHRuqrRhY9u+JDkFEpFVTzUJERKJSshARkaiULEREJColCxERiUrJQkREolKyEBGRqJQsREQkKiULERGJSslCRESi0h3c0mRn9ema6BBEpIWpZiEiIlEpWYiISFRKFiIiEpWShYiIRKVkISIiUSlZiIhIVEoWIiISlZKFiIhEpWQhIiJRKVmIiEhUShYiIhKVkoWIiESlZCEiIlGZu8dv42aTgceAZGCmuz9QZ/mjwCXhx05AT3fPDpdVAivCZVvc/ZrG9jVu3DhfunTpScV505NvAfDbfzr/pNYXEWmrzOw9dx8XrVzchig3s2TgceAKoAhYYmaz3f3D6jLufmdE+a8AYyM2cczdC+IVn4iIxC6ezVDnAevcfYO7lwGzgGsbKX8z8Js4xiMiIicpnsmiL7A14nNROO8EZjYAGAS8EjE7w8yWmtnbZnZd/MIUEZFo4vmkPKtnXkMdJNOB5929MmJef3ffbmaDgVfMbIW7r6+1A7MZwAyA/v37N0fMIiJSj3jWLIqAfhGf84DtDZSdTp0mKHffHr5vAF6jdn9GdZmn3H2cu4/Lzc1tjphFRKQe8UwWS4BhZjbIzNIIEsLsuoXM7AwgB3grYl6OmaWH0z2AicCHddcVEZGWEbdmKHevMLM7gHkEl87+zN0/MLP7gKXuXp04bgZmee1reM8EnjSzKoKE9kDkVVQiItKy4tlngbvPBebWmXd3nc/31LPeX4HR8YxNRERipzu4RUQkKiULERGJSslCRESiimufRVuhMaFERBqnmoWIiESlZCEiIlEpWYiISFRKFiIiEpWShYiIRKVkISIiUSlZiIhIVEoWIiISlZKFiIhEZbVHBm+7zGwPsPkUNtED2NtM4TQnxdU0iqtpFFfTtMe4Brh71KfHtZtkcarMbKm7j0t0HHUprqZRXE2juJqmI8elZigREYlKyUJERKJSsjjuqUQH0ADF1TSKq2kUV9N02LjUZyEiIlGpZiEiIlF12GRhZjea2QdmVmVmDV5FYGaTzWy1ma0zs2+2QFzdzOxlM1sbvuc0UK7SzArD1+w4xtPo9zezdDP7bbj8HTMbGK9YmhDTF8xsT8Tx+WK8Ywr3+zMz221mKxtYbmb2ozDu983s7FYS18VmdjDieN3dQnH1M7NXzWxV+Lf41XrKtPgxizGuFj9mZpZhZu+a2fIwrnvrKRO/v0d375Av4EzgDOA1YFwDZZKB9cBgIA1YDpwV57geAr4ZTn8TeLCBcodb4BhF/f7APwNPhNPTgd+2gpi+APw4Af+nJgFnAysbWD4F+AtgwATgnVYS18XAnAQcrz7A2eF0FrCmnn/LFj9mMcbV4scsPAZdwulU4B1gQp0ycft77LA1C3df5e6roxQ7D1jn7hvcvQyYBVwb59CuBZ4Np58Frovz/hoTy/ePjPd54DIzswTHlBDuvgjY30iRa4FfeOBtINvM+rSCuBLC3Xe4+7Jw+mNgFdC3TrEWP2YxxtXiwmNwOPyYGr7qdjrH7e+xwyaLGPUFtkZ8LiL+/2l6ufsOCP7TAj0bKJdhZkvN7G0zi1dCieX715Rx9wrgINA9TvHEGhPADWGzxfNm1i+O8TRFIv4/xer8sHnjL2Y2sqV3HjaXjCX4tRwpoceskbggAcfMzJLNrBDYDbzs7g0er+b+e0xpjo20Vma2AOhdz6Jvu/sfY9lEPfNO+fKxxuJqwmb6u/t2MxsMvGJmK9x9/anGVkcs3z8ux6gRsezvT8Bv3L3UzL5E8Evr0jjGFKuWPlaxWkYw5MNhM5sC/AEY1lI7N7MuwAvA19z9UN3F9azSIscsSlwJOWbuXgkUmFk28KKZjXL3yL6ouB2vdp0s3P3yU9xEERD5qzQP2H6K22w0LjPbZWZ93H1HWN3e3cA2tofvG8zsNYJfP82dLGL5/tVliswsBTiN+DZ5RI3J3fdFfHwaeDCO8TRFXP4/narIE6G7zzWz/zWzHu4e9zGQzCyV4IT8a3f/fT1FEnLMosWVyGMW7rM4/LufDEQmi7j9PaoZqnFLgGFmNsjM0gg6jOJ25VFoNvB34fTfASfUgMwsx8zSw+kewETgwzjEEsv3j4z3M8ArHvauxUnUmOq0aV9D0ObcGswGbg2v8JkAHKxuckwkM+td3a5tZucRnBf2Nb5Ws+zXgJ8Cq9z9hw0Ua/FjFktciThmZpYb1igws0zgcuCjOsXi9/fYkr35rekFTCXIwqXALmBeOP90YG5EuSkEV0OsJ2i+indc3YGFwNrwvVs4fxwwM5y+AFhBcCXQCuAf4hjPCd8fuA+4JpzOAH4HrAPeBQa3wDGKFtN/AR+Ex+dVYEQL/Z/6DbADKA//b/0D8CXgS+FyAx4P415BA1fhJSCuOyKO19vABS0U14UETSTvA4Xha0qij1mMcbX4MQPGAH8L41oJ3B3Ob5G/R93BLSIiUakZSkREolKyEBGRqJQsREQkKiULERGJSslCRESiUrIQaQIzOxy9VKPrPx/edY+ZdTGzJ81sfTiK6CIzG29maeF0u75pVtoWJQuRFhKOH5Ts7hvCWTMJ7q4d5u4jCUbL7eHBAIkLgZsSEqhIPZQsRE5CeEfxw2a20sxWmNlN4fykcOiHD8xsjpnNNbPPhKt9jvCOfDMbAowHvuPuVRAM3eLufw7L/iEsL9IqqJorcnKuBwqAfKAHsMTMFhEMvTIQGE0wYvAq4GfhOhMJ7qYGGAkUejAwXH1WAufGJXKRk6CahcjJuZBgZNtKd98FvE5wcr8Q+J27V7n7ToLhRqr1AfbEsvEwiZSZWVYzxy1yUpQsRE5OQw+UaexBM8cIxu6BYFyhfDNr7G8wHSg5idhEmp2ShcjJWQTcFD6MJpfg0aXvAosJHryUZGa9CB6/WW0VMBTAg2ePLAXujRi9dJiZXRtOdwf2uHt5S30hkcYoWYicnBcJRv9cDrwC/EfY7PQCwciuK4EnCZ6wdjBc58/UTh5fJHgI1jozW0Hw7I3qZzVcAsyN71cQiZ1GnRVpZmbWxYMnqHUnqG1MdPed4TMIXg0/N9SxXb2N3wPf8ujPiRdpEboaSqT5zQkfUpMG3B/WOHD3Y2b2XYLnJG9paOXwoU5/UKKQ1kQ1CxERiUp9FiIiEpWShYiIRKVkISIiUSlZiIhIVEoWIiISlZKFiIhE9f8BxgtVmtqDamQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8b9c51de48>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    plt.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的logloss。\n",
    "可以看出在训练集上C越大（正则越少）的模型性能越好；\n",
    "但在测试集上当C=0.1时性能最好（L2正则）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
